
function trim(s)
	return (s:gsub("^%s*(.-)%s*$", "%1"))
end

function split(s)
	local ret = {}
	for k,v in string.gmatch(s, "[^\t]+") do
		ret[#ret+1] = trim(k)
	end
	return ret
end

os.execute(
	"cat " .. arg[1] .. " |" ..
	" sed 's/  / /g' |" .. 
	" sed 's/ \\([0-9]\\+\\) :: /\\t\\1\\t/g' |" .. 
	" sed 's/ \\([0-9]\\+\\);/\\t\\1\\t/g' |" ..
	" sed 's/ /_/g' |" ..
	" sed 's/^\\([^\\t.]\\+\\)./\\1\\t/g' |" ..
	" tr '[:upper:]' '[:lower:]' " ..
	" > /tmp/temp1")

local f = io.open(arg[1]..'.dat', 'w')
for line in io.lines('/tmp/temp1') do
	if line:len() > 0 then
		local toks = split(line)
		f:write(toks[1] .. '\t' .. toks[3] .. '\t')
		for i = 4,#toks,2 do
			if string.find(toks[i],"_") == nil then
				f:write(toks[i] .. '\t' .. toks[i+1] .. '\t')
			end
		end
		f:write('\n')
	end
end
f:close()

os.execute("cat /tmp/temp2 | sed 's/\\t[0-9]\\+//g' | awk '{ for(i=1;i<=NF;i++) print $i}' | sort | uniq > " .. arg[2])
